import htmllib, formatter, urllib
# This Python file uses the following encoding: utf-8
import os, sys
from HTMLParser import HTMLParser
    
class ParserOrkutCommunities(htmllib.HTMLParser): # derive new HTML parser

    def __init__(self) :        # class constructor
        htmllib.HTMLParser.__init__(self, formatter.NullFormatter())  # base class constructor
        self.output = open ('communities_output.html', 'a');
        self.curr_class = [""]
        self.tape = ""
        self.communities = []
        self.localComunidade = ""
        self.localDescricao = ""
        


    def start_h3(self, attrs):
        if(len(attrs) > 0) :
            put = False
            for attr in attrs:
                if(attr[0].strip() == "class" and attr[1].strip() == "smller"):
                    self.curr_class.append(attr[1].strip())
                    put = True
            if not put: self.curr_class.append("none")
        else: self.curr_class.append("none")

    def end_h3(self):
        if(self.curr_class) : self.curr_class.pop()

    def start_a(self, attrs):
        if("smller" in self.curr_class):
            self.curr_class.append("a")

    def end_a(self):
        if(self.curr_class) : self.curr_class.pop()

    def start_div(self, attrs):
        if(len(attrs) > 0) :
            put = False
            for attr in attrs:
                if(attr[0].strip() == "class" and attr[1].strip() == "para"):
                    self.curr_class.append(attr[1].strip())
                    put = True
            if not put: self.curr_class.append("none")
        else: self.curr_class.append("none")

    def end_div(self):
        if(self.localComunidade and self.localDescricao):
            self.communities.append((self.localComunidade,self.localDescricao))
            self.localComunidade = ""
            self.localDescricao = ""
        if(self.curr_class) : self.curr_class.pop()

    def handle_data(self, data):
        if self.savedata is not None:
            self.savedata = self.savedata + data
        else:
           if self.nofill:
                self.formatter.add_literal_data(data)
           else:
                self.formatter.add_flowing_data(data)

        data = data.strip()

        if data:            
            if(self.is_last_attribute("a", self.curr_class)):
                self.output.write("\nCOMUNIDADE: " + data + "\n")
                self.localComunidade = data
            if(self.is_last_attribute("para", self.curr_class)):
                    self.output.write("Desc >>> " + data + "\n")
                    self.localDescricao = data

            self.tape = ""

    def is_last_attribute(self, attr, list):
        return attr == self.get_last_attribute(list)

    def get_last_attribute(self, list):
        index = len(list)-1
        has_last = False
        while (not has_last) and index >=0:
            if list[index] != "none":
                return list[index]
                has_last = True
            index -= 1
        if not has_last:
            return ""

    def get_communities(self):
        return self.communities 

class ParserOrkutScrapbook(htmllib.HTMLParser): # derive new HTML parser

    def __init__(self) :        # class constructor
        htmllib.HTMLParser.__init__(self, formatter.NullFormatter())
        self.output = open ('scrapbook_output.html', 'a');
        self.curr_class = [""]
        self.next_links = [""]
        self.next_link = ""
        self.tape = ""
        self.found_gt = False
        self.scrapBooks = []
        self.localTape = ""
        self.localData = ""
        self.uid = ""

    def start_span(self, attrs):
        if(len(attrs) > 0) :
            put = False
            for attr in attrs:
                if(attr[0] == "class" and attr[1]=="rf"):
                    self.curr_class.append(attr[1])
                    put = True
            if not put: self.curr_class.append("none")
        else: self.curr_class.append("none")
    
    def end_span(self):
        if(self.curr_class) : self.curr_class.pop()
    
    def start_a(self, attrs):
        if("rf" in self.curr_class):            
            self.curr_class.append("a")
            if(len(attrs)>0):
                for attr in attrs:
                    if(attr[0]=="href"):               
                        self.next_links.append(attr[1])
        if("smller" in self.curr_class):
            self.curr_class.append("a")
            if(len(attrs)>0):
                for attr in attrs:
                    if(attr[0]=="href"):
                        self.uid = attr[1][18:]
    
    def start_div(self, attrs):
        if(len(attrs) > 0) :
            put = False          
            for attr in attrs:
                if(attr[0].strip() == "class" and attr[1].strip() == "para"):
                    self.curr_class.append(attr[1].strip())
                    put = True
            if not put: self.curr_class.append("none")
        else: self.curr_class.append("none")
    
    def end_div(self):
        if(self.localData or self.localTape):
            self.scrapBooks.append((self.uid,self.localTape,self.localData))
            self.localData = ""
            self.localTape = ""
            self.uid = "" 
        if(self.curr_class) : self.curr_class.pop()    
    
    def start_img(self, attrs):
        if(len(attrs) > 0) :
            put = False          
            for attr in attrs:
                if(attr[0] == "class" and attr[1] == "listimg"):
                    self.curr_class.append(attr[1])                    
                    put = True    
                if(attr[0] == "title" and "listimg" in self.curr_class):
                    self.tape += attr[1]
            if not put: self.curr_class.append("none")
        else: self.curr_class.append("none")
    
    def start_h3(self, attrs):
        if(len(attrs) > 0) :
            put = False          
            for attr in attrs:
                if(attr[0].strip() == "class" and attr[1].strip() == "smller"):
                    self.curr_class.append(attr[1].strip())
                    put = True
            if not put: self.curr_class.append("none")
        else: self.curr_class.append("none")
    
    def end_h3(self):
        if(self.curr_class) : self.curr_class.pop()
        
    
        
    def handle_data(self, data):
        if self.savedata is not None:
            self.savedata = self.savedata + data            
        else:
           if self.nofill:
                self.formatter.add_literal_data(data)
           else:
                self.formatter.add_flowing_data(data)                
                
        data = data.strip()   
        
        if data:           
            
            if("listimg" in self.curr_class):
                if(self.tape.strip()):
                    self.output.write("SCRAP DE: " + self.tape + "\n")
                    self.localTape = self.tape
                    if(self.curr_class) : self.curr_class.pop()
            if(self.is_last_attribute("para", self.curr_class)):
                    self.output.write(">>> " + data + "\n")
                    self.localData = data
            if(self.is_last_attribute("a", self.curr_class)):
                if(len(self.next_links)>0):
                    link = self.next_links.pop()
                    if(data == "next" and (not self.found_gt)):                        
                        self.next_link = link
                        self.found_gt = True                        
                    if(self.curr_class) : self.curr_class.pop()
            
            self.tape = ""
            
            
    def is_last_attribute(self, attr, list):
        return attr == self.get_last_attribute(list)
            
    def get_last_attribute(self, list):
        index = len(list)-1
        has_last = False
        while (not has_last) and index >=0:
            if list[index] != "none":
                return list[index]
                has_last = True
            index -= 1 
        if not has_last:
            return ""
    
    def get_next_link(self):
        return self.next_link
    def has_next_link(self):
        return self.next_link != ""
    def get_scrapbooks(self):
        return self.scrapBooks

class ParserOrkutProfile(htmllib.HTMLParser): # derive new HTML parser
    
    def __init__(self) :        # class constructor
        htmllib.HTMLParser.__init__(self, formatter.NullFormatter())  # base class constructor
        self.output = open ('profile_output.html', 'a');
        self.links = []        # create an empty list for storing links
        self.curr_class = [""]        
        self.curr_tag = ""
        self.tags_used = ["div", "a"]
        self.p_classes = ["username", "sml", "usericon", "listfl", "listp"] 
        self.a_names = ["birthday, age, aboutme, sexualorientation, drinking", "hometown",
                        "passions", "sports", "music", "movies", "cuisines", "email", "city",
                        "state", "country", "turnoffs", "fticlw", "imbywf", "looks", "occupation",
                        "careerinterests"]
        
        self.testimonials=[]
        self.counter=0
        self.curr_name = ""
        self.curr_testimonial = ""
        self.curr_uid = ""
        self.get_data = False
              
        self.tape = ""

    
    
    def handle_starttag(self, tag, method, attrs):
        method(attrs)
        if((len(attrs) > 0) and (not tag in self.tags_used)):
            has_put = False
            for attr in attrs:
                if tag == "p":
                    if(attr[0] == "class" and attr[1] in self.p_classes):
                        self.curr_class.append(attr[1])                        
                        has_put = True
                    
            if not has_put:
                self.curr_class.append("none")            
        elif(not tag in self.tags_used):
            self.curr_class.append("none")
        #condicao para testes
        """if((len(attrs) > 0)):
            for attr in attrs:
                if tag == "a":
                    if(attr[0] == "name" and attr[1]=="birthday"):
                        print self.get_last_attribute(self.curr_class)"""
            
        
    def handle_endtag(self, tag, method):
        method()
        if(self.curr_class):
            self.curr_class.pop()
                 
    def end_p(self):
        if self.get_last_attribute(self.curr_class) in self.p_classes: self.curr_class.pop()
            
    """def start_p(self, attrs):
        if(len(attrs) > 0) :
            has_put = False
            for attr in attrs:
                if(attr[0] == "class" and attr[1] in self.p_classes):                    
                    self.curr_class.append(attr[1])
                    has_put = True
            if not has_put:
                self.curr_class.append("none")
        else:
            self.curr_class.append("none")
            
    def end_p(self):    
        if self.get_last_attribute(self.curr_class) in self.p_classes :
            self.curr_class.pop()"""
    
    
    def start_a(self, attrs) :  # override handler of <A ...>...</A> tags
        # process the attributes
        if len(attrs) > 0 :
            put = False
            init_length = len(self.curr_class)
            for attr in attrs :
                for name in self.a_names:
                    if(attr[0]=="name" and attr[1] == name):
                        self.curr_class.append(attr[1])
                        put = True
                        self.tape += name.upper() + " :::::::::::"
                        
                if((not put) and self.is_last_attribute("para", self.curr_class)):
                    self.get_data = True
                    self.curr_uid = attr[1][18:]
                
                if(len(self.curr_class) == init_length): self.curr_class.append("none")
        else: self.curr_class.append("none")
            
                    
    def end_a(self):
        #if self.get_last_attribute(self.curr_class) in self.a_names : self.curr_class.pop()
        if(self.curr_class):
            #print self.get_last_attribute(self.curr_class) 
            self.curr_class.pop()
    
    def start_div(self, attrs):#se for um <div class="userinfo">, muda o curr_class pra o handle_data tratar o dado corretamente
        if(len(attrs) > 0) :
            put = False          
            for attr in attrs:
                if(attr[0] == "class" and attr[1] == "userinfo"):
                    self.curr_class.append(attr[1])
                    put = True
                
                if(attr[0] == "id" and attr[1] == "app_content_testimonials"):                    
                    self.tape += "TESTIMONIAIS PARA ESSE USUARIO: \n"
                    self.curr_class.append(attr[1])
                    put = True
                if(attr[0] == "class" and attr[1] == "para"): 
                    self.tape += "TESTIMONIAL: "
                    self.curr_class.append(attr[1])
            if not put: self.curr_class.append("none")
        else: self.curr_class.append("none")
                
    def end_div(self):
        if(self.counter> 0):             
            self.testimonials.append((self.curr_uid, self.curr_name, self.curr_testimonial))
        self.counter = 0     
        self.curr_name = ""
        self.curr_testimonial = ""
        self.curr_uid = ""
        if(self.curr_class): self.curr_class.pop()
            
    def end_table(self):
        if(self.curr_class): self.curr_class.pop()
        
    def end_img(self):
        if(self.curr_class): self.curr_class.pop()
    def end_b(self):
        if(self.curr_class): self.curr_class.pop()
        
    def handle_data(self, data):
        if self.savedata is not None:
            self.savedata = self.savedata + data            
        else:
           if self.nofill:
                self.formatter.add_literal_data(data)
           else:
                self.formatter.add_flowing_data(data)                
                
        data = data.strip()   
        
        if data:            
            if("userinfo" in self.curr_class):  
                self.output.write("DADOS DO USUARIO: \n")                                
                if(self.is_last_attribute("username", self.curr_class)):                   
                    self.output.write("Nome do usuario: " + data + "\n")            
                if(self.is_last_attribute("sml", self.curr_class)):
                    self.output.write("SML do usuario: " + data + "\n")
            if("app_content_testimonials" in self.curr_class):
                if("para" in self.curr_class):
                    if(self.counter==0):
                        self.curr_name = data
                        
                    if(self.counter == 1): self.curr_testimonial += data[2:]
                    
                    if(self.counter >= 2):
                        self.curr_testimonial += data
                    
                    self.counter +=1     
                self.output.write(self.tape + data + "\n")
                
            if(self.is_last_attribute("a", self.curr_class)):
                if(self.get_data):
                    self.curr_name = data
                    self.get_data = False
            
            if(self.is_last_attribute("listfl", self.curr_class)):
                self.output.write(self.tape + data + "\n")
            if(self.is_last_attribute("listp", self.curr_class)):
                self.output.write(self.tape + data + "\n")            
            if(self.is_last_attribute("birthday", self.curr_class)):
                self.output.write(self.tape + data + "\n")
            if(self.is_last_attribute("age", self.curr_class)):
                self.output.write(self.tape + data + "\n")
            if(self.is_last_attribute("aboutme", self.curr_class)):         
                self.output.write(self.tape + data + "\n")
            if(self.is_last_attribute("sexualorientation", self.curr_class)):         
                self.output.write(self.tape + data + "\n")
            if(self.is_last_attribute("drinking", self.curr_class)):                
                self.output.write(self.tape + data + "\n")
            self.tape = ""
    
    def remove_last_on_group(self, group):
        found_item = -1
        index = len(self.curr_class)-1
        while found_item < 0 & index >= 0:
            if(self.curr_class[index] in group):
                self.curr_class.pop(index)
                found_item = 1    
            index -= 1
            
    def is_last_attribute(self, attr, list):
        return attr == self.get_last_attribute(list)
            
    def get_last_attribute(self, list):
        index = len(list)-1
        has_last = False
        while (not has_last) and index >=0:
            if list[index] != "none":
                return list[index]
                has_last = True
            index -= 1 
        if not has_last:
            return ""
    def get_testimonials(self):
        return self.testimonials

    
class ParserOrkutFriends(htmllib.HTMLParser): # derive new HTML parser
    
    def __init__(self) :        # class constructor
        htmllib.HTMLParser.__init__(self, formatter.NullFormatter())
        self.output = open ('friends_output.html', 'a');
        self.next_links = [""]
        self.next_link = ""
        self.curr_class = [""]        
        self.dataurl = 'C:\\Temp\\leo.html'
        self.tape = ""
        self.curr_uid = "" 
        self.curr_friendname = ""
        self.friends = []
        self.found_gt = False
        
    def start_span(self, attrs):
        if(len(attrs) > 0) :
            put = False          
            for attr in attrs:
                if(attr[0] == "class" and attr[1] == "nor"):
                    self.curr_class.append(attr[1])
                    put = True
                if(attr[0] == "class" and attr[1]=="rf"):
                    self.curr_class.append(attr[1])
                    put = True
            if not put: self.curr_class.append("none")
        else: self.curr_class.append("none")
    
    def end_span(self):
        if(self.curr_class) : self.curr_class.pop()
    
    def start_h3(self, attrs):
        if(len(attrs) > 0) :
            put = False          
            for attr in attrs:
                if(attr[0] == "class" and attr[1] == "smller"):
                    self.curr_class.append(attr[1])
                    put = True
            if not put: self.curr_class.append("none")
        else: self.curr_class.append("none")
        
    def end_span(self):
        if(self.curr_class) : self.curr_class.pop()
        
    def start_a(self, attrs):
        if("rf" in self.curr_class):            
            self.curr_class.append("a")
            if(len(attrs)>0):
                for attr in attrs:
                    if(attr[0]=="href"):               
                        self.next_links.append(attr[1])
        if("smller" in self.curr_class):            
            self.curr_class.append("a")
            if(len(attrs)>0):
                for attr in attrs:
                    if(attr[0]=="href" and attr[1][1] == "P"):
                        uid = ""
                        if(len(attr[1])>18): uid = attr[1][18:]               
                        self.curr_uid = uid
        
        
    def start_img(self, attrs):
        if(len(attrs) > 0) :
            put = False          
            for attr in attrs:
                if(attr[0] == "class" and attr[1] == "listimg"):
                    self.curr_class.append(attr[1])
                    put = True    
                if(attr[0] == "title" and "listimg" in self.curr_class):                    
                    self.tape += attr[1]
            if not put: self.curr_class.append("none")
        else: self.curr_class.append("none")
    
        
    def handle_data(self, data):
        if self.savedata is not None:
            self.savedata = self.savedata + data            
        else:
           if self.nofill:
                self.formatter.add_literal_data(data)
           else:
                self.formatter.add_flowing_data(data)                
                
        data = data.strip()
        if data:            
            #print self.get_last_attribute(self.curr_class)
            if(self.is_last_attribute("listimg", self.curr_class)):
                if(self.tape.strip()):
                    self.output.write("AMIGO DO CARA: " + self.tape + "\n")
                    self.curr_friendname = self.tape                    
            if(self.is_last_attribute("nor", self.curr_class)):
                    self.output.write("-> " + data + "\n")
            if(self.is_last_attribute("a", self.curr_class)):
                if(len(self.next_links)>0):
                    link = self.next_links.pop()
                    if(data == ">" and (not self.found_gt)):                        
                        self.next_link = link
                        self.found_gt = True
                
                if("smller" in self.curr_class):
                    if(self.tape): self.friends.append( (self.curr_uid, self.tape) )
                
                                        
                if(self.curr_class) : self.curr_class.pop()
                
                
            self.tape = ""
            
    def is_last_attribute(self, attr, list):
        return attr == self.get_last_attribute(list)
            
    def get_last_attribute(self, list):
        index = len(list)-1
        has_last = False
        while (not has_last) and index >=0:
            if list[index] != "none":
                return list[index]
                has_last = True
            index -= 1 
        if not has_last:
            return ""
    def get_next_link(self):
        return self.next_link
    
    def has_next_link(self):
        return self.next_link != ""
    def get_friends(self):
        return self.friends
